import numpy as np
import pandas as pd
import seaborn as sns
import plotly.express as px
import plotly.graph_objects as go
from scipy.optimize import minimize
from sklearn import metrics

Excercise 1

df = pd.read_csv('ex2/ex2data1.txt', sep=",", header=None)
df.columns = ['exam1','exam2','admission']

fig = px.scatter(df,x='exam1',y='exam2',color='admission',symbol='admission',
          color_continuous_scale=px.colors.sequential.Jet)
fig.update_layout(legend=dict(
    yanchor="top",
    y=0.99,
    xanchor="left",
    x=0.01
))
fig.update(layout_coloraxis_showscale=False)
fig.show()

We can clearly see that this dataset is devisable into two groups

First we will implement logistic regression by hand

And afterwards with prebuilt libraries

def sigmoid(z):
    return 1 / (1 + np.exp(-z) )

y = df.iloc[:,-1].to_numpy()
x = df.iloc[:,:-1]

m, n = x.shape

x = np.concatenate([np.ones((m, 1)), x], axis=1)

def costFunction(theta, X, y):
    
    m = y.size

    J = 0
    grad = np.zeros(theta.shape)


    h = sigmoid(X.dot(theta.T))


    J =-(1/m) * (y.dot(np.log(h))+(1-y).dot(np.log(1-h)))
    grad = (1/m)  * (h-y).dot(X)
    
    return J, grad

initial_theta = np.array([0,0,0])
res = minimize(costFunction,
                        initial_theta,
                        (x, y),
                        jac=True,
                        method='TNC',
                        options={'maxiter': 10000})
cost = res.fun

theta = res.x

x_ = np.linspace(x[:,1:].min(), x[:,1:].max(),2)
y_ = -(theta[0] + x_*theta[1])/theta[2]

fig.add_trace(
    go.Scatter(
        x=x_,
        y=y_,
        mode='lines',
        line={"color":"black"},
         name="matrix_solution"
    ))

fig.show()

df['predicted']=sigmoid(x.dot(theta)).round(0)

metrics.confusion_matrix(df['admission'],df['predicted'])

array([[34,  6],
       [ 5, 55]], dtype=int64)

confusion = pd.pivot_table(df.groupby(['admission','predicted']).count()['exam1'].to_frame(),index='admission',columns='predicted',)

Scikit solution

from sklearn.linear_model import LogisticRegression

m = LogisticRegression()
m.fit(df.iloc[:,:2],df.iloc[:,-1])

LogisticRegression()

y_sci = -(m.intercept_ + x_*m.coef_[0][0])/m.coef_[0][1]

fig.add_trace(
    go.Scatter(
        x=x_,
        y=y_sci,
        mode='lines',
        line={"color":"green"},
        name="plotly_solution"
    ))
fig.update_layout(legend=dict(
    orientation="h",
    yanchor="bottom",
    y=1.02,
    xanchor="right",
    x=1
))
fig.show()

Excercise 2

df2 = pd.read_csv('ex2/ex2data2.txt', sep=",", header=None)
df2.columns = ['test1','test2','result']

df2.sample(5)

	test1	test2	result
50	0.13767	0.575290	1
89	-0.38076	0.918860	0
36	-0.23675	-0.214180	1
99	0.82316	0.663010	0
108	-0.61118	-0.067982	0

fig2 = px.scatter(df2,x='test1',y='test2',color='result',symbol='result',
          color_continuous_scale=px.colors.sequential.Jet,
                 width=600, height=600)
fig2.update_layout(legend=dict(
    yanchor="top",
    y=0.99,
    xanchor="left",
    x=0.01
))
fig2.update(layout_coloraxis_showscale=False)
fig2.show()

from sklearn.preprocessing import PolynomialFeatures    

pol_6 = PolynomialFeatures(6)

feature_df = pd.DataFrame(pol_6.fit_transform(df2.iloc[:,:-1]))
y = df2.iloc[:,-1]

def costFunction_reg(theta, X, y, lamb = 1):
    
    m = y.size

    J = 0
    grad = np.zeros(theta.shape)


    h = sigmoid(X.dot(theta.T))

    temp_theta = theta
    temp_theta[0] = 0 # we do not regularize bias
        
    J =-(1/m) * (y.dot(np.log(h))+(1-y).dot(np.log(1-h))) + (lamb/(2*m))*np.sum(np.square(temp_theta))
    
    grad = (1/m)  * (h-y).dot(X) + (lamb/m) * temp_theta
    
    return J, grad

initial_theta = np.zeros(feature_df.shape[1])

X = feature_df
cost, grad = costFunction_reg(initial_theta, X, y, 1)
cost

0.6931471805599453

def generate_solution_with_lambda(lambda_ = 0):

    initial_theta = np.zeros(X.shape[1])
    options= {'maxiter': 500}

    res = minimize(costFunction_reg,
                            initial_theta,
                            (X, y, lambda_),
                            jac=True,
                            method='TNC',
                            options=options)

    cost = res.fun

    theta = res.x
    return cost, theta

def plot_solution(df,theta):
    df_ = df
    x_min, x_max = min(df_['test1']), max(df_['test1'])
    y_min, y_max = min(df_['test2']), max(df_['test2'])

    j = np.linspace(x_min, x_max, 100)
    k = np.linspace(y_min, y_max, 100)

    z = np.zeros((j.size, k.size))
    # Evaluate z = theta*x over the grid
    for i, ji in enumerate(j):
        for ii, kii in enumerate(k):
            z[i, ii] = np.dot(pol_6.fit_transform(np.array([[ji,kii]])), theta)

    z = z.T
    
    return j,k,z

fig3 = go.Figure(fig2)
lambdas = [1]
for l in lambdas:
    c,t = generate_solution_with_lambda(l)
    j,k,z = plot_solution(df2,t)
    
    fig3.add_trace(
        go.Contour(
            z=z,
            x=j,
            y=k,
            contours=dict(
                start=0.5,
                end=0.5,
                size=1,
            ),
            contours_coloring='lines',
            line_width=2,
            colorscale='Reds',
            name = f'Lambda: {l}'
        )
)
fig3.update_layout(legend=dict(
    orientation="h",
    yanchor="bottom",
    y=1.02,
    xanchor="right",
    x=1
))

fig3.show()

Scikit solution

elastic_log_reg = LogisticRegression(penalty='elasticnet',solver = 'saga',max_iter=500,l1_ratio=0.5,fit_intercept=False)

clf = elastic_log_reg.fit(feature_df,y)

clf.score(X,y)

0.8305084745762712

sci_theta = clf.coef_[0]

fig3 = go.Figure(fig2)
j,k,z = plot_solution(df2,sci_theta)
fig3.add_trace(
    go.Contour(
        z=z,
        x=j,
        y=k,
        contours=dict(
            start=0.5,
            end=0.5,
            size=1,
        ),
        contours_coloring='lines',
        line_width=2,
        colorscale='Reds',
        name = 'Scikit boundry'
    )
)
fig3.update_layout(legend=dict(
orientation="h",
yanchor="bottom",
y=1.02,
xanchor="right",
x=1
))
# fig3.data = (fig3.data[1],fig3.data[0])
fig3.show()